# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
import random
import logging
import requests
from ganji_zf.user_agents import user_agent_list


class RandomUserAgentMiddleware(object):
    def process_request(self, request, spider):
        request.headers['User-Agent'] = random.choice(user_agent_list)
        if not request.headers.get('Referer'):
            request.headers['Referer'] = "http://wh.ganji.com/"


class ProxyMiddleware(object):
    def __init__(self, proxy_url):
        self.logger = logging.getLogger(__name__)
        self.proxy_url = proxy_url

    def get_random_proxy(self):
        try:
            response = requests.get(self.proxy_url)
            if response.status_code == 200:
                proxy = response.text
                return proxy
        except requests.ConnectionError:
            return False

    def process_request(self, request, spider):
        proxy = self.get_random_proxy()
        if proxy:
            uri = 'http://{proxy}'.format(proxy=proxy)
            self.logger.debug('使用代理 ' + proxy)
            request.meta['proxy'] = uri

    def process_response(self, request, response, spider):
        '''对返回的response处理'''
        # 如果返回的response状态不是200，重新生成当前request对象
        if response.status != 200:
            proxy = self.get_random_proxy()
            uri = 'http://{proxy}'.format(proxy=proxy)
            # 对当前reque加上代理
            self.logger.debug('返回状态码:' + str(response.status))
            self.logger.debug('响应repsonse_url:' + response.url)
            self.logger.debug('请求request_url:' + request.url)
            self.logger.debug('请求头referer:' + request.headers['Referer'].decode())
            request.meta['proxy'] = uri
            request.headers['User-Agent'] = random.choice(user_agent_list)
            request.dont_filter = True
            return request
        return response

    def process_exception(self, request, exception, spider):
        # 出现异常时（超时）使用代理
        self.logger.debug('代理超时')
        proxy = self.get_random_proxy()
        uri = 'http://{proxy}'.format(proxy=proxy)
        request.meta['proxy'] = uri
        request.headers['User-Agent'] = random.choice(user_agent_list)
        request.dont_filter = True
        return request

    @classmethod
    def from_crawler(cls, crawler):
        settings = crawler.settings
        return cls(
            proxy_url=settings.get('PROXY_URL')
        )
